package com.jiuzhou.crawlers.taobao.crawler.jd;

import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.collections.bag.SynchronizedSortedBag;

import com.jiuzhou.crawlers.taobao.pipeline.jd.JdGoodsPipeline;
import com.jiuzhou.crawlers.taobao.processor.jd.JdPagePreocessor;
import com.jiuzhou.crawlers.taobao.processor.jd.JdPricePreocessor;
import com.jiuzhou.crawlers.taobao.util.UrlMakerUtil;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.downloader.Downloader;
import us.codecraft.webmagic.downloader.selenium.SeleniumDownloader;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.QueueScheduler;

public class JdClawerStart implements PageProcessor  {
    private Site site =  Site.me().setRetryTimes(3).setSleepTime(1000);  //站点设置
    static Map<String,String> idNameMap = new HashMap<String, String>();
    static Set<String> uri = new HashSet<String>();
    public static int  pageId = 10;
	public static final String URL_LIST = "http://list\\.jd\\.com/list\\.html\\?cat=670,671,672&page=\\d+\\&trans=1\\&JL=6_0_0";  //cat为分类标识，该分类为笔记本
	public Site getSite() {
		// TODO Auto-generated method stub
	    return site;
	}

	 
	  public void process(Page page) {
		  String goodsUrlId= "";
	       if (page.getHtml().links().regex(URL_LIST).match()) {
           page.putField("title", page.getHtml().css("div.items").xpath("//dt/a/text()").all().toString());
           page.putField("subtitle", page.getHtml().css("div.items").xpath("//dd/a/text()").all().toString());
           page.putField("Url", page.getHtml().css("div.items").xpath("//dt/a/@href").all().toString());
           page.putField("subUrl", page.getHtml().css("div.items").xpath("//dd/a/@href").all().toString());
           page.putField("pageRe", page.getHtml().css("pre").toString());
           page.putField("gid", page.getHtml().css("li.gl-item").xpath("//div[@class='gl-i-wrap j-sku-item']/@data-sku").all().toString());
           List<String> ids =page.getHtml().css("li.gl-item").xpath("//div[@class='gl-i-wrap j-sku-item']/@data-sku").all();
 		         for (String id : ids) {
 		        	 goodsUrlId = goodsUrlId+"J_"+id+",";
 		         }
 		   String text = JdPricePreocessor.running("https://p.3.cn/prices/mgets?skuIds="+goodsUrlId);//根据商品ID爬取价格
 		   String idss = page.getHtml().css("li.gl-item").xpath("//div[@class='gl-i-wrap j-sku-item']/@data-sku").all().toString();
 		   page.putField("price", text);
           page.putField("name", page.getHtml().css("div.p-name").xpath("//em/text()").all().toString());
           page.putField("length", ids.size());
	      page.addTargetRequest("http://list.jd.com/list.html?cat=670,671,672&page="+pageId);
	       pageId++;
	       }
	       //System.out.println(page.getHtml().links().regex(URL_LIST).all().toString());

      }
	public  static void main(String[] args) {
	
		  String list = "http://list.jd.com//list.html?cat=670,671,672&amp;page=7&amp;trans=1&amp;JL=6_0_0&amp;ms=6";
		     Spider.create(new JdClawerStart()).addUrl(list)
		    .setDownloader(new SeleniumDownloader("C:\\chromedriver\\chromedriver.exe"))
	         .addPipeline(new JdGoodsPipeline("hdfs://master:9000/user/yanganzhen/JD/"))
	       .run();
		// TODO Auto-generated method stub
//        Spider.create(new ClawerStart()) .addUrl("https://www.jd.com/allsort.aspx")
//        .addPipeline(new JdGoodsPipeline("hdfs://master:9000/user/yanganzhen/test/"))
//        //开启5个线程抓取
//        .thread(5)
//        //启动爬虫
//        .run();
	}



}
